From d73b185634c189fb3fc575a463c4bc651cc801a4 Mon Sep 17 00:00:00 2001 From: "kaf24@firebug.cl.cam.ac.uk" Date: Thu, 31 Mar 2005 13:12:29 +0000 Subject: [PATCH] bitkeeper revision 1.1236.1.174 (424bf73dRPXTiuGP-qQmSHwGKGOyRQ) Improved TLB flushing of subsets of CPUs. Can now do remote invlpg as well as complete flush. Signed-off-by: Keir Fraser --- xen/arch/x86/mm.c | 31 ++++++++++++++------- xen/arch/x86/mtrr/generic.c | 4 +-- xen/arch/x86/smp.c | 15 +++++++---- xen/arch/x86/x86_32/mm.c | 4 +-- xen/arch/x86/x86_64/mm.c | 4 +-- xen/common/grant_table.c | 25 +++++++++-------- xen/include/asm-x86/flushtlb.h | 49 +++++++++++++--------------------- xen/include/asm-x86/page.h | 13 --------- 8 files changed, 70 insertions(+), 75 deletions(-) diff --git a/xen/arch/x86/mm.c b/xen/arch/x86/mm.c index 946ea05e03..a5648e68d5 100644 --- a/xen/arch/x86/mm.c +++ b/xen/arch/x86/mm.c @@ -1476,23 +1476,34 @@ int do_mmuext_op( break; case MMUEXT_INVLPG_LOCAL: - __flush_tlb_one(op.linear_addr); + local_flush_tlb_one(op.linear_addr); break; case MMUEXT_TLB_FLUSH_MULTI: - flush_tlb_mask(d->cpuset); /* XXX KAF XXX */ - break; - case MMUEXT_INVLPG_MULTI: - flush_tlb_mask(d->cpuset); /* XXX KAF XXX */ + { + unsigned long inset = op.cpuset, outset = 0; + while ( inset != 0 ) + { + unsigned int vcpu = find_first_set_bit(inset); + inset &= ~(1UL<exec_domain[vcpu]) != NULL) ) + outset |= 1UL << ed->processor; + } + if ( op.cmd == MMUEXT_TLB_FLUSH_MULTI ) + flush_tlb_mask(outset & d->cpuset); + else + flush_tlb_one_mask(outset & d->cpuset, op.linear_addr); break; + } case MMUEXT_TLB_FLUSH_ALL: flush_tlb_mask(d->cpuset); break; case MMUEXT_INVLPG_ALL: - flush_tlb_mask(d->cpuset); /* XXX KAF XXX */ + flush_tlb_one_mask(d->cpuset, op.linear_addr); break; case MMUEXT_FLUSH_CACHE: @@ -2029,10 +2040,10 @@ int do_update_va_mapping(unsigned long va, percpu_info[cpu].deferred_ops &= ~DOP_FLUSH_TLB; break; case UVMF_INVLPG_LOCAL: - __flush_tlb_one(va); + local_flush_tlb_one(va); break; case UVMF_INVLPG_ALL: - flush_tlb_mask(d->cpuset); /* XXX KAF XXX */ + flush_tlb_one_mask(d->cpuset, va); break; } @@ -2317,7 +2328,7 @@ void ptwr_flush(const int which) /* Ensure that there are no stale writable mappings in any TLB. */ /* NB. INVLPG is a serialising instruction: flushes pending updates. */ - __flush_tlb_one(l1va); /* XXX Multi-CPU guests? */ + local_flush_tlb_one(l1va); /* XXX Multi-CPU guests? */ PTWR_PRINTK("[%c] disconnected_l1va at %p now %p\n", PTWR_PRINT_WHICH, ptep, pte); @@ -2636,7 +2647,7 @@ int ptwr_do_page_fault(unsigned long addr) likely(!shadow_mode_enabled(ed->domain)) ) { *pl2e = mk_l2_pgentry(l2e & ~_PAGE_PRESENT); - flush_tlb(); /* XXX Multi-CPU guests? */ + local_flush_tlb(); /* XXX Multi-CPU guests? */ } /* Temporarily map the L1 page, and make a copy of it. */ diff --git a/xen/arch/x86/mtrr/generic.c b/xen/arch/x86/mtrr/generic.c index 26000d4b49..e8d990aec7 100644 --- a/xen/arch/x86/mtrr/generic.c +++ b/xen/arch/x86/mtrr/generic.c @@ -261,7 +261,7 @@ static void prepare_set(void) } /* Flush all TLBs via a mov %cr3, %reg; mov %reg, %cr3 */ - __flush_tlb(); + local_flush_tlb(); /* Save MTRR state */ rdmsr(MTRRdefType_MSR, deftype_lo, deftype_hi); @@ -273,7 +273,7 @@ static void prepare_set(void) static void post_set(void) { /* Flush TLBs (no need to flush caches - they are disabled) */ - __flush_tlb(); + local_flush_tlb(); /* Intel (P6) standard MTRRs */ wrmsr(MTRRdefType_MSR, deftype_lo, deftype_hi); diff --git a/xen/arch/x86/smp.c b/xen/arch/x86/smp.c index 962417238f..023b8fcab5 100644 --- a/xen/arch/x86/smp.c +++ b/xen/arch/x86/smp.c @@ -148,17 +148,20 @@ static inline void send_IPI_allbutself(int vector) } static spinlock_t flush_lock = SPIN_LOCK_UNLOCKED; -static unsigned long flush_cpumask; +static unsigned long flush_cpumask, flush_va; asmlinkage void smp_invalidate_interrupt(void) { ack_APIC_irq(); perfc_incrc(ipis); - local_flush_tlb(); + if ( flush_va == FLUSHVA_ALL ) + local_flush_tlb(); + else + local_flush_tlb_one(flush_va); clear_bit(smp_processor_id(), &flush_cpumask); } -void flush_tlb_mask(unsigned long mask) +void __flush_tlb_mask(unsigned long mask, unsigned long va) { ASSERT(local_irq_is_enabled()); @@ -172,6 +175,7 @@ void flush_tlb_mask(unsigned long mask) { spin_lock(&flush_lock); flush_cpumask = mask; + flush_va = va; send_IPI_mask(mask, INVALIDATE_TLB_VECTOR); while ( flush_cpumask != 0 ) cpu_relax(); @@ -190,6 +194,7 @@ void new_tlbflush_clock_period(void) spin_lock(&flush_lock); flush_cpumask = (1UL << smp_num_cpus) - 1; flush_cpumask &= ~(1UL << smp_processor_id()); + flush_va = FLUSHVA_ALL; send_IPI_allbutself(INVALIDATE_TLB_VECTOR); while ( flush_cpumask != 0 ) cpu_relax(); @@ -203,13 +208,13 @@ void new_tlbflush_clock_period(void) static void flush_tlb_all_pge_ipi(void *info) { - __flush_tlb_pge(); + local_flush_tlb_pge(); } void flush_tlb_all_pge(void) { smp_call_function(flush_tlb_all_pge_ipi, 0, 1, 1); - __flush_tlb_pge(); + local_flush_tlb_pge(); } void smp_send_event_check_mask(unsigned long cpu_mask) diff --git a/xen/arch/x86/x86_32/mm.c b/xen/arch/x86/x86_32/mm.c index ac1f10def5..c33758b69d 100644 --- a/xen/arch/x86/x86_32/mm.c +++ b/xen/arch/x86/x86_32/mm.c @@ -48,7 +48,7 @@ int map_pages( { /* Super-page mapping. */ if ( (l2_pgentry_val(*pl2e) & _PAGE_PRESENT) ) - __flush_tlb_pge(); + local_flush_tlb_pge(); *pl2e = mk_l2_pgentry(p|flags|_PAGE_PSE); v += 1 << L2_PAGETABLE_SHIFT; @@ -66,7 +66,7 @@ int map_pages( } pl1e = l2_pgentry_to_l1(*pl2e) + l1_table_offset(v); if ( (l1_pgentry_val(*pl1e) & _PAGE_PRESENT) ) - __flush_tlb_one(v); + local_flush_tlb_one(v); *pl1e = mk_l1_pgentry(p|flags); v += 1 << L1_PAGETABLE_SHIFT; diff --git a/xen/arch/x86/x86_64/mm.c b/xen/arch/x86/x86_64/mm.c index d5e9254925..2d19d6f85f 100644 --- a/xen/arch/x86/x86_64/mm.c +++ b/xen/arch/x86/x86_64/mm.c @@ -89,7 +89,7 @@ int map_pages( { /* Super-page mapping. */ if ( (l2_pgentry_val(*pl2e) & _PAGE_PRESENT) ) - __flush_tlb_pge(); + local_flush_tlb_pge(); *pl2e = mk_l2_pgentry(p|flags|_PAGE_PSE); v += 1 << L2_PAGETABLE_SHIFT; @@ -107,7 +107,7 @@ int map_pages( } pl1e = l2_pgentry_to_l1(*pl2e) + l1_table_offset(v); if ( (l1_pgentry_val(*pl1e) & _PAGE_PRESENT) ) - __flush_tlb_one(v); + local_flush_tlb_one(v); *pl1e = mk_l1_pgentry(p|flags); v += 1 << L1_PAGETABLE_SHIFT; diff --git a/xen/common/grant_table.c b/xen/common/grant_table.c index fceee60984..7cd543b834 100644 --- a/xen/common/grant_table.c +++ b/xen/common/grant_table.c @@ -102,7 +102,8 @@ __gnttab_map_grant_ref( if ( ((host_virt_addr != 0) || (flags & GNTMAP_host_map) ) && unlikely(!__addr_ok(host_virt_addr))) { - DPRINTK("Bad virtual address (%x) or flags (%x).\n", host_virt_addr, flags); + DPRINTK("Bad virtual address (%x) or flags (%x).\n", + host_virt_addr, flags); (void)__put_user(GNTST_bad_virt_addr, &uop->handle); return GNTST_bad_gntref; } @@ -332,8 +333,10 @@ __gnttab_map_grant_ref( */ } - /* Only make the maptrack live _after_ writing the pte, in case - * we overwrite the same frame number, causing a maptrack walk to find it */ + /* + * Only make the maptrack live _after_ writing the pte, in case we + * overwrite the same frame number, causing a maptrack walk to find it. + */ ld->grant_table->maptrack[handle].domid = dom; ld->grant_table->maptrack[handle].ref_and_flags = (ref << MAPTRACK_REF_SHIFT) | (flags & MAPTRACK_GNTMAP_MASK); @@ -364,13 +367,14 @@ gnttab_map_grant_ref( unsigned long va = 0; for ( i = 0; i < count; i++ ) - if ( __gnttab_map_grant_ref(&uop[i], &va) == 0) + if ( __gnttab_map_grant_ref(&uop[i], &va) == 0 ) flush++; + /* XXX KAF: I think we are probably flushing too much here. */ if ( flush == 1 ) - __flush_tlb_one(va); + flush_tlb_one_mask(current->domain->cpuset, va); else if ( flush != 0 ) - local_flush_tlb(); + flush_tlb_mask(current->domain->cpuset); return 0; } @@ -457,7 +461,7 @@ __gnttab_unmap_grant_ref( unsigned long _ol1e; pl1e = &linear_pg_table[l1_linear_offset(virt)]; - + if ( unlikely(__get_user(_ol1e, (unsigned long *)pl1e) != 0) ) { DPRINTK("Could not find PTE entry for address %x\n", virt); @@ -526,14 +530,13 @@ gnttab_unmap_grant_ref( unsigned long va = 0; for ( i = 0; i < count; i++ ) - if ( __gnttab_unmap_grant_ref(&uop[i], &va) == 0) + if ( __gnttab_unmap_grant_ref(&uop[i], &va) == 0 ) flush++; if ( flush == 1 ) - __flush_tlb_one(va); - + flush_tlb_one_mask(current->domain->cpuset, va); else if ( flush != 0 ) - local_flush_tlb(); + flush_tlb_mask(current->domain->cpuset); return 0; } diff --git a/xen/include/asm-x86/flushtlb.h b/xen/include/asm-x86/flushtlb.h index 5958b5f524..8f48465cb2 100644 --- a/xen/include/asm-x86/flushtlb.h +++ b/xen/include/asm-x86/flushtlb.h @@ -77,46 +77,35 @@ static inline unsigned long read_cr3(void) /* Write pagetable base and implicitly tick the tlbflush clock. */ extern void write_cr3(unsigned long cr3); -/* - * TLB flushing: - * - * - flush_tlb() flushes the current mm struct TLBs - * - flush_tlb_all() flushes all processes TLBs - * - flush_tlb_pgtables(mm, start, end) flushes a range of page tables - * - * ..but the i386 has somewhat limited tlb flushing capabilities, - * and page-granular flushes are available only on i486 and up. - */ - -#define __flush_tlb() \ +#define local_flush_tlb() \ do { \ unsigned long cr3 = read_cr3(); \ write_cr3(cr3); \ } while ( 0 ) -#ifndef CONFIG_SMP +#define local_flush_tlb_pge() \ + do { \ + __pge_off(); \ + local_flush_tlb(); \ + __pge_on(); \ + } while ( 0 ) -#define flush_tlb() __flush_tlb() -#define flush_tlb_all() __flush_tlb() -#define flush_tlb_all_pge() __flush_tlb_pge() -#define local_flush_tlb() __flush_tlb() -#define flush_tlb_cpu(_cpu) __flush_tlb() -#define flush_tlb_mask(_mask) __flush_tlb() -#define try_flush_tlb_mask(_mask) __flush_tlb() +#define local_flush_tlb_one(__addr) \ + __asm__ __volatile__("invlpg %0": :"m" (*(char *) (__addr))) -#else +#define flush_tlb_all() flush_tlb_mask((1 << smp_num_cpus) - 1) +#ifndef CONFIG_SMP +#define flush_tlb_all_pge() local_flush_tlb_pge() +#define flush_tlb_mask(_mask) local_flush_tlb() +#define flush_tlb_one_mask(_mask,_v) local_flush_tlb_one(_v) +#else #include - -extern int try_flush_tlb_mask(unsigned long mask); -extern void flush_tlb_mask(unsigned long mask); +#define FLUSHVA_ALL (~0UL) extern void flush_tlb_all_pge(void); - -#define flush_tlb() __flush_tlb() -#define flush_tlb_all() flush_tlb_mask((1 << smp_num_cpus) - 1) -#define local_flush_tlb() __flush_tlb() -#define flush_tlb_cpu(_cpu) flush_tlb_mask(1 << (_cpu)) - +extern void __flush_tlb_mask(unsigned long mask, unsigned long va); +#define flush_tlb_mask(_mask) __flush_tlb_mask(_mask,FLUSHVA_ALL) +#define flush_tlb_one_mask(_mask,_v) __flush_tlb_mask(_mask,_v) #endif #endif /* __FLUSHTLB_H__ */ diff --git a/xen/include/asm-x86/page.h b/xen/include/asm-x86/page.h index 78e2648b54..58f67d88b8 100644 --- a/xen/include/asm-x86/page.h +++ b/xen/include/asm-x86/page.h @@ -78,8 +78,6 @@ extern root_pgentry_t idle_pg_table[ROOT_PAGETABLE_ENTRIES]; extern void paging_init(void); -/* Flush global pages as well. */ - #define __pge_off() \ do { \ __asm__ __volatile__( \ @@ -94,17 +92,6 @@ extern void paging_init(void); : : "r" (mmu_cr4_features) ); \ } while ( 0 ) - -#define __flush_tlb_pge() \ - do { \ - __pge_off(); \ - __flush_tlb(); \ - __pge_on(); \ - } while ( 0 ) - -#define __flush_tlb_one(__addr) \ - __asm__ __volatile__("invlpg %0": :"m" (*(char *) (__addr))) - #endif /* !__ASSEMBLY__ */ -- 2.30.2